#!/usr/bin/env python
# -*- coding:utf-8 -*-

'''
Created on 2011-9-19
@author: zhongfeng
'''

import os,re,sys
import hashlib
import operator
import shutil
from ConfigParser import ConfigParser
from decimal import Decimal
import MySQLdb
from pageparser import ParserUtils

#DB parameter

strHost = 'localhost'
strDB = 'bigo_db_new'
strUser = 'root'
strPasswd = ''

def seEncode(ustr, encoding='utf-8'):
    '''负责把入数据库的字符串，转化成utf-8编码'''
    if ustr is None:
        return ''
    if isinstance(ustr, unicode):
        return ustr.encode(encoding, 'ignore')
    elif isinstance(ustr, (list,tuple,set)):
        return '[%s]' % ','.join([seEncode(s,encoding) for s in ustr])        
    return str(ustr)

def getDigit(s):
    s = s.replace(u',', '')
    regx = u'[0-9]+.[0-9]+|[0-9]+'
    p = re.compile(regx)
    sd = p.search(s)
    if sd is None:
        return 0
    return sd.group()

#connect to DB
def getConnect(db=strDB, host=strHost, user=strUser, passwd=strPasswd, charset="utf8"):
    return MySQLdb.connect(host=strHost, db=strDB, user=strUser, passwd=strPasswd, charset="utf8")

def initClientEncode(conn):
    '''mysql client encoding=utf8'''
    curs = conn.cursor()
    curs.execute("SET NAMES utf8")
    conn.commit()
    return curs

#===============================================================================
# 基本vo类定义
#===============================================================================  
        
class CatBaseConfig(object):
    ''' 表  cat_base_config 对应的vo class'''  
    def __init__(self,id,tProdInfo,tProdPriceCur,tProdPriceHis):
        attrsFromDict(locals())
    
    def __str__(self):
        return str(vars(self))
    
    __repr__ =__str__   
    
class ProdBaseInfo(object):
    '''商品基本信息表对应于数据里的prod_base_info_XX这样的表'''
    
    __slots__ = ('site_id','raw_id','name','url','img_url','repu','eval_num',
                 'cat_id','self_cat_id','cat_base_id','u_time')
    def __init__(self):
        pass
    
    def __str__(self):
        return os.linesep.join([seEncode((name,getattr(self,name))) for name in self.__slots__])
    
    __repr__ =__str__
    
class ProdPriceInfo(object):
    '''商品基本信息表对应于数据里的prod_price_info_的表'''
    
    __slots__ = ('id','prod_id','real_price','cur_price','m_price','diff_price','trans_price',
                 'other_dis','adwords','coupon','ex_gift','order_cut','crash_cut','u_time')

    def __str__(self):
        return os.linesep.join([seEncode((name,getattr(self,name))) for name in self.__slots__])
    
    __repr__ =__str__

#===============================================================================
#  表 `websit_base_info` db 操作
#===============================================================================
def getAllWebsiteBaseInfo():
    '''获取所有的站点的en_name与id信息'''
    conn = getConnect()
    curs = initClientEncode(conn) 
    sqlStr = 'SELECT en_name,id FROM `websit_base_info` ' #生成sql语句
    curs.execute(sqlStr)
    result = curs.fetchall()
    curs.close()
    conn.close()
    return result

#===============================================================================
#  表 `prod_db_stat` db 操作
#===============================================================================
def getProdStatInfo(site_id,cat_base_id):
    '''各站点的数据更新状态'''
    conn = getConnect()
    curs = initClientEncode(conn) 
    sqlStr = 'SELECT u_time FROM `prod_db_stat` where website_id=%s and cat_base_id=%s ' 
    param = (site_id,cat_base_id)
    curs.execute(sqlStr,param)
    result = curs.fetchone()
    curs.close()
    conn.close()
    if result:
        return result[0]
    
def saveProdStat(site_id,cat_base_id,u_time):
    conn = getConnect()
    curs = initClientEncode(conn)
    sqlStr = 'INSERT INTO `prod_db_stat`(`website_id`,`cat_base_id`,`u_time`) VALUES (%s,%s,%s)'
    param = (site_id,cat_base_id,u_time)
    curs.execute(sqlStr, param)
    conn.commit()
    curs.close()
    
def updateProdStat(site_id,cat_base_id,u_time):
    conn = getConnect()
    curs = initClientEncode(conn)
    sqlStr = 'update `prod_db_stat` set u_time=%s where website_id=%s and cat_base_id=%s'
    param = (u_time,site_id,cat_base_id)
    curs.execute(sqlStr, param)
    conn.commit()
    curs.close()
    
#===============================================================================
# 表 `prod_catagory` db proc
#===============================================================================
def getCatIdFromRawCatID(raw_cat_id, site_id):
    '''获取分类id'''
    conn = getConnect()
    curs = initClientEncode(conn) 
    sqlStr = 'SELECT id,self_cat_id,cat_base_id FROM `prod_catagory` where raw_cat_id = %s and site_id = %s'
    param = (raw_cat_id, site_id)
    curs.execute(sqlStr, param)
    result = curs.fetchone()
    curs.close()
    conn.close()
    if result != None:
        return result


#===============================================================================
#  表 `cat_base_config` db 操作
#===============================================================================
def getAllCatBaseConfig():
    '''获取所有的站点的en_name与id信息'''
    conn = getConnect()
    curs = initClientEncode(conn) 
    sqlStr = '''SELECT main_cat_id, baseinfo_table_name, priceinfo_cur_table_name, priceinfo_his_table_name,
                                   en_name  FROM `cat_base_config`''' #生成sql语句
    curs.execute(sqlStr)
    result = curs.fetchall()
    curs.close()
    conn.close()
    return result

#===============================================================================
# 商品基础信息入库函数，根据不同的类别保存进不同的表中
#===============================================================================
def saveProdBaseInfo(conn, table_name, pBaseInfo):
    '''保存商品基本信息到表 table_name,'''
    curs = initClientEncode(conn)
    sqlPattern = '''INSERT INTO {table_name} (`site_id` ,`raw_id` ,`name` ,`url` ,`img_url` , `repu`, `eval_num`,
                        `cat_id` , `self_cat_id`,`u_time` ) VALUES (%s , %s, %s, %s, %s, %s , %s, %s , %s, %s)'''
    sqlStr = sqlPattern.format(table_name=table_name)
    
    param = [seEncode(pt) for pt in (pBaseInfo.site_id , pBaseInfo.raw_id , pBaseInfo.name , pBaseInfo.url ,
                                      pBaseInfo.img_url , pBaseInfo.repu, pBaseInfo.eval_num, pBaseInfo.cat_id,
                                           pBaseInfo.self_cat_id , pBaseInfo.u_time)]
    
    curs.execute(sqlStr, param)
    conn.commit()
    ret = curs.lastrowid
    curs.close()
    return int(ret) 

def batchSaveProdBaseInfo(conn, table_name, prodList):
    ''' 批量保存商品基本信息到表 table_name'''
    curs = initClientEncode(conn)
    sqlPattern = '''INSERT INTO {table_name} (`site_id` ,`raw_id` ,`name` ,`url` ,`img_url` , `repu`, `eval_num`, 
                    `cat_id` , `self_cat_id`,`u_time` ) VALUES (%s , %s, %s, %s, %s, %s , %s, %s , %s, %s)'''
    sqlStr = sqlPattern.format(table_name=table_name)
    wparams = list()
    for pBaseInfo in prodList:
        wparams.append([seEncode(pt) for pt in (pBaseInfo.site_id , pBaseInfo.raw_id , pBaseInfo.name , pBaseInfo.url ,
                                      pBaseInfo.img_url , pBaseInfo.repu, pBaseInfo.eval_num, pBaseInfo.cat_id,
                                           pBaseInfo.self_cat_id , pBaseInfo.u_time)])
    curs.executemany(sqlStr, wparams)
    conn.commit()
    curs.close() 
    
def updateProdBaseInfo(conn, table_name, pBaseInfo):
    '''更新基本信息table_name,'''
    curs = initClientEncode(conn)
    sqlPattern = '''update {table_name} set `repu`=%s, `eval_num`=%s, url=%s,img_url=%s where site_id=%s and raw_id=%s'''
    sqlStr = sqlPattern.format(table_name=table_name)
    
    param = [seEncode(pt) for pt in (pBaseInfo.repu, pBaseInfo.eval_num,pBaseInfo.url ,
                                      pBaseInfo.img_url ,pBaseInfo.site_id,pBaseInfo.raw_id)]
    
    curs.execute(sqlStr, param)
    conn.commit()
    curs.close()

def batchUpdateProdBaseInfo(conn, table_name, prodList):
    ''' 批量更新商品基本信息到表 table_name'''
    curs = initClientEncode(conn)
    sqlPattern = '''update {table_name} set `repu`=%s, `eval_num`=%s, url=%s,img_url=%s where site_id=%s and raw_id=%s'''
    sqlStr = sqlPattern.format(table_name=table_name)
    wparams = list()
    for pBaseInfo in prodList:
        wparams.append([seEncode(pt) for pt in (pBaseInfo.repu, pBaseInfo.eval_num,pBaseInfo.url,
                                      pBaseInfo.img_url ,pBaseInfo.site_id,pBaseInfo.raw_id)])
    curs.executemany(sqlStr, wparams)
    conn.commit()
    curs.close() 
    
def getAllRawProdIds(table_name, site_id):
    '''在table_name中获取某一个站点的所有prod_id'''
    conn = getConnect()
    curs = initClientEncode(conn) 
    sqlPattern = 'SELECT raw_id,id FROM {table_name} where site_id = %s'
    sqlStr = sqlPattern.format(table_name=table_name)
    param = (site_id)
    curs.execute(sqlStr, param)
    result = curs.fetchall()
    curs.close()
    conn.close()
    return result
    
#===============================================================================
# 保存价格数据到各类别的cur_price里
#===============================================================================
def saveProdPriceInfo(conn, table_name, priceInfo):
    curs = initClientEncode(conn)
    sqlPattern = '''INSERT INTO {table_name} (`prod_id` ,`real_price` ,`cur_price` ,`diff_price`,
                 `adwords` ,`coupon` ,`ex_gift` ,`order_cut` ,`crash_cut`,`u_time` ,`u_flag`) VALUES 
                 (%s, %s, %s,%s, %s, %s,%s, %s, %s,%s,1)'''
    sqlStr = sqlPattern.format(table_name=table_name)
    param = [seEncode(pt) for pt in (priceInfo.prod_id, priceInfo.real_price, priceInfo.cur_price,
                priceInfo.diff_price, priceInfo.adwords, priceInfo.coupon, priceInfo.ex_gift,
                 priceInfo.order_cut,priceInfo.crash_cut, priceInfo.u_time)]
    curs.execute(sqlStr, param)
    conn.commit()
    ret = curs.lastrowid
    curs.close()
    return int(ret)

def updateProdPriceInfo(conn, table_name, priceInfo):
    curs = initClientEncode(conn)
    sqlPattern = ''' update {table_name} set `real_price`=%s ,`cur_price`=%s ,`diff_price`=%s, `adwords`=%s,`coupon`=%s ,`ex_gift`=%s ,
                      `order_cut`=%s ,`crash_cut`=%s ,`u_time`=%s ,`u_flag`=1 where prod_id = %s '''
    sqlStr = sqlPattern.format(table_name=table_name)
    param = [seEncode(pt) for pt in (priceInfo.real_price, priceInfo.cur_price,
                priceInfo.diff_price, priceInfo.adwords, priceInfo.coupon, priceInfo.ex_gift,
                 priceInfo.order_cut,priceInfo.crash_cut, priceInfo.u_time,priceInfo.prod_id)]
    curs.execute(sqlStr, param)
    conn.commit()
    ret = curs.lastrowid
    curs.close()
    return int(ret)

def batchSaveProdPriceInfo(conn, table_name , params):
    curs = initClientEncode(conn)
    sqlPattern = '''INSERT INTO {table_name} (`prod_id` ,`real_price` ,`cur_price` ,`m_price` ,`diff_price` ,`trans_price`,`other_dis` ,
                 `adwords` ,`coupon` ,`ex_gift` ,`order_cut` ,`crash_cut`,`u_time`,,`u_flag` ) VALUES (%s, %s, %s, %s, %s, %s,%s, %s, %s,%s, %s, %s,%s,1)''' 
    sqlStr = sqlPattern.format(table_name=table_name)
    
    wparams = list()
    for param in params:
        wparams.append([seEncode(pt) for pt in param])
    curs.executemany(sqlStr, wparams)
    conn.commit()
    curs.close()

def savePriceInfo2HisTable(conn,his_table,cur_table,baseinfo_table,site_id):
    curs = initClientEncode(conn)
    try:
        #>= date_format( %s, '%%Y-%%m-%%d' ) 
        sqlPattern = ''' insert into {his_price_table} (`prod_id` ,`real_price` ,`cur_price` ,`m_price` ,`diff_price` ,`trans_price` ,
                   `other_dis` ,`adwords` ,`coupon` ,`ex_gift` ,`order_cut` ,`crash_cut` ,`u_time` ) SELECT t1.`prod_id` , t1.`real_price` , 
                    t1.`cur_price` , t1.`m_price` , t1.`diff_price` , t1.`trans_price` , t1.`other_dis` , t1.`adwords` , t1.`coupon` , 
                    t1.`ex_gift` , t1.`order_cut` , t1.`crash_cut` , t1.`u_time` 
                    FROM {cur_price_table} AS t1 JOIN {baseinfo_table} AS t2 ON t1.prod_id = t2.id
                    WHERE t1.u_flag = 1 AND t2.site_id =%s'''
    
        sqlStr = sqlPattern.format(his_price_table=his_table,
                               cur_price_table=cur_table,baseinfo_table=baseinfo_table)

        param  = (site_id)
        curs.execute(sqlStr,param)
        
        sqlPat2 = ''' UPDATE {cur_price_table} SET `u_flag` = '0' WHERE `u_flag` = 1 '''
        sqlStr2 = sqlPat2.format(cur_price_table=cur_table)
        curs.execute(sqlStr2)
        conn.commit()
    except Exception:
        conn.rollback()
    finally:
        curs.close()
    
def getCurPriceByProdId(conn, table_name , prod_id):
    curs = initClientEncode(conn)
    sqlPattern = ''' select real_price,cur_price from {table_name} where prod_id = %s '''
    sqlStr = sqlPattern.format(table_name=table_name)
    param = (prod_id)
    curs.execute(sqlStr,param)
    result = curs.fetchone()
    curs.close()
    return result    

# exceptions
class LogFormatException(Exception):
    """All work requests have been processed."""
    pass

def attrsFromDict(d):
    self = d.pop('self')
    for k,v in d.iteritems():
        setattr(self, k, v)

class Singleton(object):
    ''' python 风格的单例模式 '''
    def __new__(cls,*args,**kargs):
        if '_inst' not in vars(cls):
            cls._inst = super(Singleton,cls).__new__(cls, *args,**kargs)
        return cls._inst

class SiteNameIDDictFactory(Singleton): 
    '''根据站点name获取对应的id,对应表web_site_info中的数据'''
    _t_site_dict = None
   
    @classmethod
    def getSiteIdByName(cls, siteName):
        if cls._t_site_dict is None:
            cls._t_site_dict = dict(getAllWebsiteBaseInfo())
        return cls._t_site_dict[siteName]


class CatBaseConfigDictFactory(Singleton):
    ''' Config表的列结构 ，main_cat_id, baseinfo_table_name, 
        priceinfo_cur_table_name, priceinfo_his_table_name,en_name '''
    _t_cbconfig_dict = None
    
    @classmethod
    def _initDict(cls):
        configs = getAllCatBaseConfig()
        for c in configs:
            cls._t_cbconfig_dict[str(c[0])] = c
    
    @classmethod
    def getConfigById(cls, catBaseId):
        if cls._t_cbconfig_dict is None:
            cls._t_cbconfig_dict = {}
            cls._initDict()
        return cls._t_cbconfig_dict[str(catBaseId)]
    
    @classmethod
    def getCatBaseConfig(cls,catBaseId):
        config = cls.getConfigById(catBaseId)
        return CatBaseConfig( catBaseId, config[1],config[2],config[3])

def getCatKey(url):
    '''计算caturl的md5值'''
    m2 = hashlib.md5()
    m2.update(url)
    dest2 = int(m2.hexdigest(), 16)
    return str(dest2)[0:16]

class ProdCatDictFactory(Singleton): 
    '''字典类key为:siteId_rawCatId,value是 （id,self_cat_id,cat_base_id）'''
    _t_cat_dict = {}
    
    @classmethod
    def __getKey(cls, siteId, rawCatId):
        return '_'.join(map(str, (siteId, rawCatId)))
    
    @classmethod
    def getProdCatId(cls, siteId, catUrl):
        rawCatId = getCatKey(catUrl)
        key = cls.__getKey(siteId, rawCatId)
        value = cls._t_cat_dict.get(key, None)
        if value is None:
            value = getCatIdFromRawCatID(rawCatId, siteId)
            cls._t_cat_dict[key] = value
        return value
        
class ProdBaseInfoBuilder(Singleton):
    '''构建商品基本信息类'''
    def _getCatIdFromRawInfo(self,site_id,catUrl):
        return ProdCatDictFactory.getProdCatId(site_id, catUrl)
    
    def getResult(self,linestr,site_id):
        ret = linestr.split('|')
        if len(ret) != 10:
            return
        prodBaseInfo = ProdBaseInfo()
        prodBaseInfo.site_id = site_id
        prodBaseInfo.raw_id = ret[0]
        prodBaseInfo.u_time = ret[2]
        prodBaseInfo.name = ret[3]

        evalNum = ret[4]
        if evalNum:
            evalNum = evalNum.replace(u',','')
        
        repu = float(ret[5])
        if repu > 5.1:
            repu = repu * 5 / 100
        prodBaseInfo.repu = str(repu)
        prodBaseInfo.eval_num = evalNum
        prodBaseInfo.url = ret[7]
        prodBaseInfo.img_url = ret[8]
        
        catUrl = eval(ret[-1])[0]
        ret = self._getCatIdFromRawInfo(site_id,catUrl)
        if ret:
            cat_id,self_cat_id,cat_base_id = ret
        else:
            #self.logger.info('   '.join([str(s) for s in (site_id,catUrl)]))
            return
   
        prodBaseInfo.cat_id = cat_id
        prodBaseInfo.self_cat_id = self_cat_id
        prodBaseInfo.cat_base_id = cat_base_id
        
        return prodBaseInfo
    
#===============================================================================
# Product 基础信息入库处理
#===============================================================================

def __getLine(fileName, encoding='gb18030'):
    ''' 目前的日志文件存储的字符编码为gb18030'''
    with open(fileName, 'r') as fInput:
        for line in fInput:
            yield line.strip().decode(encoding, 'ignore')

def __judgeDuplicateProd(prodIdsDict, prodBaseInfo):
    '''判断一个商品在数据库里是否已经存在,重复返回True'''
    if prodBaseInfo is None:
        return True
    raw_id = prodBaseInfo.raw_id
    if raw_id in prodIdsDict:
        return True
    else:
        prodIdsDict[raw_id] = ''  #保证文件里的同一个商品只入库一次
    return False

def __j360BuyFilterFunc(prodBaseInfo):
    ''''判断商品是否属于京东自营'''
    return not str(getattr(prodBaseInfo,'raw_id','')).startswith('100')
                            
filterFuncDict = {'6':__j360BuyFilterFunc}

def __getProdFilterFunc(site_id):
    dfltfFunc = lambda x:True
    return filterFuncDict.get(str(site_id),dfltfFunc);

def __chooseProd(prodBaseInfo,catBaseId):
    '''过滤不符合规则的商品，由于目前是按类别入库的，入库的商品，
                   首先需要在此类别中，其次要满足一些其他的规则，例如，在京东中
                    过滤所有id为100开头的非自营商品'''
    if prodBaseInfo is None:
        return False
    filterFunc = __getProdFilterFunc(prodBaseInfo.site_id)
    if (catBaseId == getattr(prodBaseInfo,'cat_base_id',None)) and filterFunc(prodBaseInfo):
        return True
    return False
    
def __getExistProdRawId(table_name,siteId):
    return dict(getAllRawProdIds(table_name,siteId))

def __singleProdLoad2Db(conn,line,site_id,catBaseId):
    ''' 商品基本信息入库 '''
    #根据类别id获取表的名字（不同的大分类入不同的表中）
    
    catBaseConfig = CatBaseConfigDictFactory.getCatBaseConfig(catBaseId)
    prodBaseInfoTable = catBaseConfig.tProdInfo
    builder = ProdBaseInfoBuilder()
    prodBaseInfo = builder.getResult(line, site_id)
    prodId = None
    if prodBaseInfo and __chooseProd(prodBaseInfo, catBaseId):
        prodId  = saveProdBaseInfo(conn, prodBaseInfoTable, prodBaseInfo)

    return prodId
   
def createProdBaseInfo(siteName, catBaseId, fileName):
    ''' 商品基本信息批量入库 '''
    conn = getConnect()
    site_id = SiteNameIDDictFactory.getSiteIdByName(siteName)
    #根据类别id获取表的名字（不同的大分类入不同的表中）
    catBaseConfig = CatBaseConfigDictFactory.getCatBaseConfig(catBaseId)
    prodBaseInfoTable = catBaseConfig.tProdInfo
    prodIdsDict = __getExistProdRawId(prodBaseInfoTable,site_id)
    prodList = []
    for line in __getLine(fileName):
        builder = ProdBaseInfoBuilder()
        prodBaseInfo = builder.getResult(line, site_id)
        if __chooseProd(prodBaseInfo, catBaseId) and \
                        not __judgeDuplicateProd(prodIdsDict, prodBaseInfo):
            print prodBaseInfo.name,prodBaseInfo.raw_id
            prodList.append(prodBaseInfo)
            if len(prodList) == 200:
                batchSaveProdBaseInfo(conn, prodBaseInfoTable, prodList)
                prodList = []
    if prodList:
        batchUpdateProdBaseInfo(conn, prodBaseInfoTable, prodList)  
    conn.close()
    
#===============================================================================
#  价格信息处理
#=============================================================================== 
def wrapDecimal(priceStr):
    priceStr = ''.join(priceStr.split())
    return Decimal(str(priceStr))

class PriceInfoBuilder(object):
    def __init__(self,line=''):
        self.line = line
    @staticmethod
    def _calRealPrice(curPrice, coupon, orderCut, crashCut):
        deIt = [wrapDecimal(p) for p in (curPrice, coupon, orderCut, crashCut)]
        return str(float(reduce(operator.sub,deIt)))   
    
    def getAdwords(self):
        return self.adWords
    
    def setAdwords(self,adwords):
        self.adWords = adwords
      
    def getCoupon(self):
        '''返券'''
        return '0.00'
    def getExGift(self):
        '''赠品'''
        return ''   
    def getOrderCut(self):
        '''下单立减'''
        return '0.00'  
    def getCrashCut(self):
        '''返现'''
        return '0.00'
   
    def getSiteDefineFinalPrice(self):
        '''网站标注的商品实际成交价（折算返券、赠品等等）'''
        return '0.00'    
    
    def getRealPrice(self,curPrice):
        '''商品计算后得到的价格'''
        fPrice = self.getSiteDefineFinalPrice() 
        if fPrice != '0.00':
            return fPrice
        return PriceInfoBuilder._calRealPrice(curPrice,
                self.getCoupon(),self.getOrderCut(),self.getCrashCut())
    
    def getResult(self,line,prod_id):
        ret = line.split('|')
        if len(ret) != 10:
            return
        prodPriceInfo = ProdPriceInfo()
        cur_price = ret[1].replace(' ','')
        prodPriceInfo.cur_price = cur_price
        prodPriceInfo.u_time  = ret[2]
        prodPriceInfo.adwords = ret[6]
        self.setAdwords(ret[6])
        prodPriceInfo.prod_id = prod_id
        prodPriceInfo.coupon  = self.getCoupon()
        prodPriceInfo.crash_cut = self.getCrashCut()
        prodPriceInfo.ex_gift = self.getExGift()
        prodPriceInfo.order_cut = self.getOrderCut()
        prodPriceInfo.real_price = self.getRealPrice(ret[1])
        return prodPriceInfo

class J360PriceInfoBuilder(PriceInfoBuilder):
    def getCoupon(self):
        p1 = re.compile(ur'([0-9]+)元?京?券')
        p2 = re.compile(ur'京?券([0-9]+)')
        ret1 = p1.search(self.adWords)
        ret = ret1 if ret1 != None else p2.search(self.adWords)
        if ret != None:
            return ret.group(1)
        else:
            return '0.00'

    def getExGift(self):
        i = self.adWords.rfind(u'赠')#//送
        if i != -1:
            ret = self.adWords[i:].replace(u'，', ' ').replace(u'！', ' ')
            ti = ret.find(' ')
            if ti != -1:
                return ret[0:ti]
            else:
                return ret
        else:
            return ''

    def getOrderCut(self):
        p1 = re.compile(ur'下单.*减([0-9]+)')
        p2 = re.compile(ur'下单直降([0-9]+)')
        ret1 = p1.search(self.adWords)
        ret = ret1 if ret1 != None else p2.search(self.adWords)
        if ret != None:
            return ret.group(1)
        else:
            return '0.00'   
    
    def getCrashCut(self):
        p1 = re.compile(ur'返现([0-9]+)')
        ret1 = p1.search(self.adWords)
        if ret1 != None:
            return ret1.group(1)
        else:
            return '0.00'
    
    def getSiteDefineFinalPrice(self):
        p1 = re.compile(ur'相当于([0-9]+)')
        p2 = re.compile(ur'成交价?([0-9]+)')
        p3 = re.compile(ur'([0-9]+)元?成交')
        ret1 = p1.search(self.adWords)
        ret2 = ret1 if ret1 != None else p2.search(self.adWords)
        ret = ret2 if ret2 != None else p3.search(self.adWords)
        if ret != None:
            return ret.group(1)
        else:
            return '0.00'

class Coo8PriceInfoBuilder(PriceInfoBuilder):
    def getExGift(self): 
        return self.adWords.split('@')[1]
    
    def getCrashCut(self):
        crashCutSeg = self.adWords.split('@')[0]
        return getDigit(crashCutSeg)
    
class GomePriceInfoBuilder(PriceInfoBuilder):
    def getCrashCut(self):
        crashCutSeg = self.adWords
        return getDigit(crashCutSeg)
    
IcsonPriceInfoBuilder = J360PriceInfoBuilder
NewEggPriceInfoBuilder = Coo8PriceInfoBuilder

class PriceInfoBuilderFactory(Singleton):
    _builder_map = {'360buy':J360PriceInfoBuilder,'coo8':Coo8PriceInfoBuilder,
                             'gome':GomePriceInfoBuilder,'icson':IcsonPriceInfoBuilder,
                             'newegg':NewEggPriceInfoBuilder}
    @classmethod
    def getPriceBuilder(cls,websiteName):
        return cls._builder_map.get(websiteName,PriceInfoBuilder) 
    
def __getRawId(line):
    ret = line.split('|')
    if len(ret) != 10:
        return
    raw_id = ret[0] 
    return raw_id  

def __getDateStrFromFileName(fName):
    p = re.compile('[0-9]{4}-[0-9]{2}-[0-9]{2}')
    ret = p.search(fName)
    if ret != None:
        return ret.group()
          
class LoaderConfig(object):
    '''load data 配置'''
    @classmethod
    def _init(cls):
        curPath = os.path.abspath(os.path.dirname(sys.argv[0]))
        fileName = os.path.join(curPath, 'load.conf')
        cls.cf = ConfigParser()  
        cls.cf.read(fileName)
        
    @classmethod
    def getConfig(cls,option):
        if not hasattr(cls, 'cf'):
            cls._init()
        try:
            return cls.cf.get('conf',option)
        except Exception:
            pass
        
    @classmethod
    def getBaseDir(cls):
        ret = cls.getConfig('base_dir')
        if ret:
            return ret
        else:
            raise Exception('No base_dir found')
    
    @classmethod
    def getBackupDir(cls):
        ret = cls.getConfig('backup_dir')
        if ret:
            return ret
        else:
            raise Exception('No backup_dir found')
    
    @classmethod
    def getSitesSet(cls):
        sites = cls.getConfig('sites')
        if sites:
            return set([site.strip() for site in sites.split(',')])
        return set()
    
    @classmethod
    def getCatId(cls):
        cat_id = cls.getConfig('cat_id')
        if cat_id:
            return int(cat_id)
        
def ld2CurPriceInfo(siteName,fileName,catBaseId,newProdSaveFlag = True):
    site_id = SiteNameIDDictFactory.getSiteIdByName(siteName)
    #根据类别id获取表的名字（不同的大分类入不同的表中）
    catBaseConfig = CatBaseConfigDictFactory.getCatBaseConfig(catBaseId)
    priceCurTable = catBaseConfig.tProdPriceCur
    prodIdDict = __getExistProdRawId(catBaseConfig.tProdInfo,site_id)
    #raw_id_set = set()
    conn = getConnect()
    for line in __getLine(fileName):
        raw_id = __getRawId(line)
        if raw_id is None:# or raw_id in raw_id_set:
            continue
        #raw_id_set.add(raw_id)
        prod_id = prodIdDict.get(raw_id, None)
        if prod_id is None and newProdSaveFlag:
            prod_id = __singleProdLoad2Db(conn,line,site_id,catBaseId)
            #print 'insert new prod %s' % prod_id
            if prod_id:#属于过滤掉的商品
                prodIdDict[raw_id] = prod_id
        if prod_id is None:
            #print 'break raw_id is  %s ' % raw_id
            continue
        PriceInfoBuilderClass = PriceInfoBuilderFactory.getPriceBuilder(siteName)
        priceInfoBuilder = PriceInfoBuilderClass()
        try:
            priceInfo = priceInfoBuilder.getResult(line,prod_id)
        except Exception ,e:
            print e
            continue
        curPriceInfo = getCurPriceByProdId(conn,priceCurTable,prod_id)
        if curPriceInfo:
            real_price,cur_price = curPriceInfo
            if wrapDecimal(real_price) != wrapDecimal(priceInfo.real_price) or  \
                              wrapDecimal(cur_price) != wrapDecimal(priceInfo.cur_price):
                diff_price = (wrapDecimal(real_price) - wrapDecimal(priceInfo.cur_price))
                if wrapDecimal(priceInfo.real_price) < Decimal('0.1') or diff_price < Decimal('0.1'):
                    continue
                priceInfo.diff_price = str(diff_price)
                print curPriceInfo,priceInfo.prod_id,priceInfo.real_price,priceInfo.cur_price
                updateProdPriceInfo(conn,priceCurTable,priceInfo)
            #else:
                #pass
                #print 'no price change'
        else:
            priceInfo.diff_price = '0.00'
            if wrapDecimal(priceInfo.real_price) > Decimal('0.1'):
                print 'save raw_id : %s price is:%s ' % (raw_id,priceInfo.real_price)
                saveProdPriceInfo(conn,priceCurTable,priceInfo)
    conn.close()
    
def ld2HisPriceInfo(catBaseId, siteName):
    ''' 把新入库的数据导入历史价格表中  '''
    site_id = SiteNameIDDictFactory.getSiteIdByName(siteName)
    catBaseConfig = CatBaseConfigDictFactory.getCatBaseConfig(catBaseId)
    priceHisTable = catBaseConfig.tProdPriceHis
    priceCurTable = catBaseConfig.tProdPriceCur
    prodBaseInfoTable = catBaseConfig.tProdInfo
    #raw_id_set = set()
    conn = getConnect()
    savePriceInfo2HisTable(conn,priceHisTable,priceCurTable,
                               prodBaseInfoTable,site_id)
    conn.close()
    
def __isAlreadyLoadDb(siteName,catBaseId,dateStr):
    '''只有比当前数据库中记录新的才可以入库，表prod_db_stat
          中记录各个类别最新的数据'''
    site_id = SiteNameIDDictFactory.getSiteIdByName(siteName)
    db_u_time = getProdStatInfo(site_id,catBaseId)
    if db_u_time and dateStr <= db_u_time.strftime('%Y-%m-%d'):
        return True
    else:
        return False
    
def updateProdStatLoadDb(siteName,catBaseId,dateStr):
    ''' 更新入库状态 '''
    site_id = SiteNameIDDictFactory.getSiteIdByName(siteName)
    db_u_time = getProdStatInfo(site_id,catBaseId)
    if db_u_time is None: 
        saveProdStat(site_id, catBaseId, dateStr)
    elif dateStr > db_u_time.strftime('%Y-%m-%d'):
        updateProdStat(site_id, catBaseId, dateStr)
        
def backuplogs(src,siteName):
    dst_base = LoaderConfig.getBackupDir()
    dst = os.path.join(dst_base,siteName)
    if not os.path.isdir(dst):
        os.mkdir(dst)
    print 'moving file from %s to %s' % (src,dst)
    shutil.move(src, dst)
    
def uProdBaseInfo(siteName, catBaseId, fileName):
    ''' 商品基本信息批量更新 '''
    conn = getConnect()
    site_id = SiteNameIDDictFactory.getSiteIdByName(siteName)
    #根据类别id获取表的名字（不同的大分类入不同的表中）
    catBaseConfig = CatBaseConfigDictFactory.getCatBaseConfig(catBaseId)
    prodBaseInfoTable = catBaseConfig.tProdInfo
    #prodIdsDict = __getExistProdRawId(prodBaseInfoTable,site_id)
    prodList = []
    for line in __getLine(fileName):
        builder = ProdBaseInfoBuilder()
        prodBaseInfo = builder.getResult(line, site_id)
        if prodBaseInfo:
            print prodBaseInfo.name,prodBaseInfo.raw_id
            prodList.append(prodBaseInfo)
            if len(prodList) == 200:
                batchUpdateProdBaseInfo(conn, prodBaseInfoTable, prodList)
                prodList = []
    if prodList:
        batchUpdateProdBaseInfo(conn, prodBaseInfoTable, prodList)      
    conn.close()
    
def loadDataToDB(files,catBaseId,beginDate = None,endDate = None):
    for fileName in files:#itertools.chain(files[1:],files[0:1]):
        fName = os.path.split(fileName)[-1]
        dateStr = __getDateStrFromFileName(fName)
        if dateStr is None:
            print 'file :%s ,no date str found.' % fileName
            continue
        if beginDate != None and dateStr != None and cmp(dateStr,beginDate) < 0:
            continue
        elif endDate != None and dateStr != None and cmp(dateStr,endDate) > 0:
            break
        siteName = fName.split('_')[0]
        if siteName in LoaderConfig.getSitesSet():
            print 'current proc file is :%s' % fileName
            if __isAlreadyLoadDb(siteName, catBaseId, dateStr):
                print '**** file :%s is already load to db *****' % fileName
                backuplogs(fileName,siteName)
                continue
            try:   
                ld2CurPriceInfo(siteName, fileName, catBaseId)
                ld2HisPriceInfo(catBaseId,siteName)
                updateProdStatLoadDb(siteName,catBaseId,dateStr)
                backuplogs(fileName,siteName)
            except Exception ,e:
                print e
        else:
            print 'site :%s is not in load set' % siteName
    

#===============================================================================
# module 测试
#===============================================================================
def testProdBaseInfoBuilder():
    testStr = '''429958|1199.00|2011-09-21 20:42:47|TCL BCD-176K50 176升 两门 冰箱（银灰色）
                  |92|1471|直降400元！冰箱销量冠军！高效压缩机|http://www.360buy.com/product/429958.html||
                      ('http://www.360buy.com/products/737-794-878.html', 3)'''
    builder = ProdBaseInfoBuilder()
    ret = builder.getResult(testStr,site_id = 6)
    print ret

def testCatBaseConfigDictFactory():
    for t in xrange(10):
        print CatBaseConfigDictFactory.getCatBaseConfig(1)

import glob
def load(rootBase):
    for rootDir in os.listdir(rootBase):
        if rootDir.find('.') >= 0:
            continue
        rootDir = os.path.join(rootBase,rootDir)
        if os.path.isdir(rootDir):
            plog = r'%s%s*log*' % (rootDir,os.sep)
            files = glob.glob(plog)
            print files
            loadDataToDB(files,catBaseId = LoaderConfig.getCatId())

if __name__ == '__main__':
    #testProdBaseInfoBuilder()
    #createProdBaseInfo(siteName = u'amazon', catBaseId = 1, fileName = r'F:\spider_exe-20111222\spider_exe\amazon\amazon_spider.log')
    #uProdBaseInfo(siteName=u'coo8',catBaseId = 1,fileName=r'F:\python_workspace\exe\coo8\log\coo8_spider.log.2011-12-29')
    #rootBase=r'I:/log'
    load(rootBase = LoaderConfig.getBaseDir())
    #ld2HisPriceInfo(1, '2011-11-13',2)
    #createProdBaseInfo(siteName = '360buy',catBaseId = 1,
    #                   fileName = r'I:\log\360buy\360buy_spider.log.2011-10-24')
    
